# TODO: predykcje na teście (ROC / confusion matrix)
# TODO: czasy uczenia w tabelce
from textwrap import wrap
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import plotly.graph_objects as go
import seaborn as sns
from sklearn import metrics
import os
def readucr(filename):
data = np.loadtxt(filename)
Y = data[:, 0]
X = data[:, 1:]
return X, Y
class ClassificationEvaluator:
"""Class to evaluate classification predictions"""
def __init__(self, predictions: pd.DataFrame):
if np.all(
np.isin(
[
"method",
"probability_prediction",
"binary_prediction",
"true_label",
],
predictions,
)
):
raise Exception(
"Not all the required columns are in the given predictions dataframe"
)
self.predictions = predictions
self.methods = predictions["method"].unique()
def calculate_classification_metrics(
self, if_return_melted_not_pivoted: bool
) -> pd.DataFrame:
"""Function to calculate metrics for given predictions
Following metrics are calculated:
- accuracy
- precision
- recall
- F1-score
- AUC
Args:
if_return_melted_not_pivoted (bool): id returned dataframe should be in melted or pivoted format
Returns:
pd.DataFrame: dataframe with metrics
"""
metric_df = pd.DataFrame()
for method in self.methods:
current_predictions = self.predictions.loc[
self.predictions.method == method
]
accuracy_score = metrics.accuracy_score(
current_predictions["true_label"], current_predictions["binary_prediction"]
)
precision_score = metrics.precision_score(
current_predictions["true_label"], current_predictions["binary_prediction"]
)
recall_score = metrics.recall_score(
current_predictions["true_label"], current_predictions["binary_prediction"]
)
f1_score = metrics.f1_score(
current_predictions["true_label"], current_predictions["binary_prediction"]
)
roc_auc_score = metrics.roc_auc_score(
current_predictions["true_label"],
current_predictions["probability_prediction"],
)
metric_df = metric_df.append(
{
"method": method,
"accuracy": accuracy_score,
"f1": f1_score,
"precision": precision_score,
"recall": recall_score,
"AUC": roc_auc_score,
},
ignore_index=True,
)
if if_return_melted_not_pivoted:
metric_df = pd.melt(
metric_df,
id_vars=["method"],
var_name="metric_name",
value_name="metric_value",
)
return metric_df
def plot_confusion_matrices_grid(
self, plot_title: str, cols_number_on_plot: int
):
"""Plot confusion matrices for given predictions
Args:
plot_title (str): Title for the plot
cols_number_on_plot (int): Numbers of the columns on the facet plot
"""
def pd_confusion_matrix(df):
return metrics.confusion_matrix(
df["true_label"],
df["binary_prediction"]
)
def draw_heatmap(*args, **kwargs):
data = kwargs.pop("data")
sns.heatmap(
data["confusion_matrix"].tolist()[0],
annot=True,
square=True,
fmt="g",
**kwargs,
)
confusion_matrix_per_group = (
self.predictions.groupby(["method"])
.apply(pd_confusion_matrix)
.reset_index(name="confusion_matrix")
)
conf_matrix_values = np.concatenate(
np.array(confusion_matrix_per_group["confusion_matrix"])
).ravel()
fg = sns.FacetGrid(
confusion_matrix_per_group, col="method", col_wrap=cols_number_on_plot
)
cbar_ax = fg.fig.add_axes([1., 0.3, 0.02, 0.4])
fg.map_dataframe(
draw_heatmap, cbar_ax=cbar_ax, vmin=0, vmax=conf_matrix_values.max()
)
fg.set_titles(col_template="{col_name}")
fg.fig.subplots_adjust(wspace=0.1, hspace=0.1)
fg.set_axis_labels("Predicted", "Actual")
fg.fig.suptitle(f"{plot_title}", y=1.05)
fg.fig.show()
def plot_roc_curves(self, plot_title: str, width: int, height: int):
"""Function to plot ROC curves
Args:
plot_title (str): Title for the plot
"""
fig = go.Figure()
colors = [
"#574368",
"#8488d3",
"#cfd3c1",
"#f8c868",
"#8ddb34",
"#69cfef",
"#d1b3ff",
"#ff8e65",
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2',
'#7f7f7f',
'#bcbd22',
'#17becf'
]
for index, method in enumerate(self.methods):
fpr, tpr, _ = metrics.roc_curve(
self.predictions.loc[
self.predictions["method"] == method, "true_label"
],
self.predictions.loc[
self.predictions["method"] == method, "probability_prediction"
],
pos_label=2
)
auc_score = metrics.auc(fpr, tpr)
fig.add_trace(
go.Scatter(
x=fpr,
y=tpr,
mode="lines",
name=f"{method} (AUC: {auc_score:.2f})",
line=dict(color=colors[index % len(colors)])
)
)
fig.update_layout(
template="plotly_white",
legend_title="<b>Model:</b>",
title_text=plot_title,
xaxis_title="FPR",
yaxis_title="TPR",
width=width,
height=height,
)
fig.show()
def plot_confusion_matrices(self):
"""Plot confusion matrices for given predictions"""
def pd_confusion_matrix(df):
return metrics.confusion_matrix(
df["true_label"],
df["binary_prediction"],
),
confusion_matrix_per_group = (
self.predictions.groupby(["method"])
.apply(pd_confusion_matrix)
.reset_index(name="confusion_matrix")
)
for index, row in confusion_matrix_per_group.iterrows():
_ = sns.heatmap(
row["confusion_matrix"], annot=True, square=True, fmt="g", cmap="YlGnBu"
)
title = "\n".join(wrap(row["method"], 20))
plt.title(title, fontsize=20, y=1.1)
plt.xlabel("Predicted value", fontsize=14)
plt.ylabel("Actual value", fontsize=14)
plt.show()
def plot_precision_recall(self, width: int, height: int):
def pd_precision_recall(df):
precision, recall, thresholds = metrics.precision_recall_curve(
df["true_label"], df["probability_prediction"], pos_label=2
)
return pd.Series(
[precision, recall, thresholds],
index=["precision", "recall", "thresholds"],
)
precision_recall_per_group = (
self.predictions.groupby(["method"])
.apply(pd_precision_recall)
.reset_index()
)
colors = [
"#574368",
"#8488d3",
"#cfd3c1",
"#f8c868",
"#8ddb34",
"#69cfef",
"#d1b3ff",
"#ff8e65",
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2',
'#7f7f7f',
'#bcbd22',
'#17becf'
]
fig = go.Figure()
for index, row in precision_recall_per_group.iterrows():
fig.add_trace(
go.Scatter(
x=row.thresholds,
y=row.recall,
mode="lines",
name=f"Recall {row.method}",
line=dict(color=colors[index % len(colors)]),
)
)
fig.add_trace(
go.Scatter(
x=row.thresholds,
y=row.precision,
name=f"Precision {row.method}",
line=dict(color=colors[index % len(colors)], dash="dot"),
)
)
fig.update_layout(
title_text="Precision/Recall curves",
template="plotly_white",
legend_title="<b>Model:</b>",
xaxis_title="Threshold value",
yaxis_title="Metric value",
width=width,
height=height,
)
fig.show()
colors = [
"#574368",
"#8488d3",
"#cfd3c1",
"#f8c868",
"#8ddb34",
"#69cfef",
"#d1b3ff",
"#ff8e65",
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2',
'#7f7f7f',
'#bcbd22',
'#17becf'
]
dpath = "../../datasets/GunPoint/GunPoint"
x_train, y_train = readucr(dpath + "_TRAIN.txt")
x_test, y_test = readucr(dpath + "_TEST.txt")
if_display = True
train_pred_df = pd.DataFrame()
test_pred_df = pd.DataFrame()
elapsed_time_dict = {}
loss_plt_dict = {}
acc_plt_dict = {}
for method_dir in os.listdir("../data"):
train_acc = pd.read_csv(os.path.join("../data", method_dir, "train_accs.txt"), header=None)
train_loss = pd.read_csv(os.path.join("../data", method_dir, "train_losses.txt"), header=None)
val_acc = pd.read_csv(os.path.join("../data", method_dir, "val_accs.txt"), header=None)
val_loss = pd.read_csv(os.path.join("../data", method_dir, "val_losses.txt"), header=None)
elapsed_time_dict[method_dir] = pd.read_csv(os.path.join("../data", method_dir, "elapsed_time.txt"), header=None).values[0][0]
train_acc = train_acc.rename(columns={0: "train_accuracy"})
train_loss = train_loss.rename(columns={0: f"train_loss"})
val_acc = val_acc.rename(columns={0: f"validation_accuracy"})
val_loss = val_loss.rename(columns={0: f"validation_loss"})
err_df = pd.concat([train_acc, train_loss, val_acc, val_loss], axis=1)
epochs = err_df.shape[0]
plot_title = method_dir
fig1 = go.Figure()
fig2 = go.Figure()
for index, col in enumerate(err_df[["train_accuracy", "validation_accuracy"]].columns):
fig1.add_trace(
go.Scatter(
x=err_df.index,
y=err_df[col],
mode="lines",
name=f"{col}",
line=dict(color=colors[index % len(colors)])
)
)
for index, col in enumerate(err_df[["train_loss", "validation_loss"]].columns):
fig2.add_trace(
go.Scatter(
x=err_df.index,
y=err_df[col],
mode="lines",
name=f"{col}",
line=dict(color=colors[index % len(colors) + 2])
)
)
fig1.update_layout(
template="plotly_white",
legend_title="<b>Measure: </b>",
title_text=plot_title,
xaxis_title="Epoch",
yaxis_title="Accuracy",
width=900,
height=800,
)
fig2.update_layout(
template="plotly_white",
legend_title="<b>Measure: </b>",
title_text=plot_title,
xaxis_title="Epoch",
yaxis_title="Loss",
width=900,
height=800,
)
acc_plt_dict[method_dir] = fig1
loss_plt_dict[method_dir] = fig2
elapsed_time_df = pd.DataFrame.from_dict(elapsed_time_dict, orient="index")
elapsed_time_df.sort_index().T
| FCN | MLP | ResNet | quantile-MinCut_GCNConv | quantile-PoC_GCNs | quantile-TopK_GIN | quantile-TopK_GraphConv | visibility-feat | visibility-no-feat | |
|---|---|---|---|---|---|---|---|---|---|
| 0 | 3461.199276 | 1682.422909 | 5233.124094 | 71.83963 | 92.190896 | 89.127614 | 38.444067 | 87.494389 | 59.103806 |
if if_display:
for k in acc_plt_dict.keys():
acc_plt_dict[k].show()
if if_display:
for k in loss_plt_dict.keys():
loss_plt_dict[k].show()
train_pred_df = pd.DataFrame()
test_pred_df = pd.DataFrame()
for method_dir in os.listdir("../data"):
train_acc = pd.read_csv(os.path.join("../data", method_dir, "train_accs.txt"), header=None)
train_loss = pd.read_csv(os.path.join("../data", method_dir, "train_losses.txt"), header=None)
val_acc = pd.read_csv(os.path.join("../data", method_dir, "val_accs.txt"), header=None)
val_loss = pd.read_csv(os.path.join("../data", method_dir, "val_losses.txt"), header=None)
try:
# collect and process data for further classification evaluation
train_pred = pd.read_csv(os.path.join("../data", method_dir, "train_pred.csv"))
test_pred = pd.read_csv(os.path.join("../data", method_dir, "test_pred.csv"))
train_pred["method"] = method_dir
test_pred["method"] = method_dir
train_pred["binary_prediction"] = ((train_pred["pred_proba1"] < 0.5).astype(int) + 1).values
test_pred["binary_prediction"] = ((test_pred["pred_proba1"] < 0.5).astype(int) + 1).values
train_pred["true_label"] = y_train.astype(int)
test_pred["true_label"] = y_test.astype(int)
train_pred["probability_prediction"] = train_pred["pred_proba2"] # pos_label=2 !!!! (see metrics.precision_recall_curve, metrics.roc_curve)
test_pred["probability_prediction"] = test_pred["pred_proba2"] # pos_label=2 !!!! (see metrics.precision_recall_curve, metrics.roc_curve)
train_pred_df = pd.concat([train_pred_df, train_pred])
test_pred_df = pd.concat([test_pred_df, test_pred])
except FileNotFoundError:
pass
excluded_methods = test_pred_df.loc[test_pred_df["probability_prediction"].isin([np.nan, np.inf, -np.inf])]["method"].unique()
excluded_methods
array([], dtype=object)
test_pred_df = test_pred_df.loc[~test_pred_df["method"].isin(excluded_methods)].reset_index(drop=True)
train_pred_df = train_pred_df.loc[~train_pred_df["method"].isin(excluded_methods)].reset_index(drop=True)
test_evaluator = ClassificationEvaluator(predictions=test_pred_df)
test_evaluator.calculate_classification_metrics(if_return_melted_not_pivoted=False)
| method | accuracy | f1 | precision | recall | AUC | |
|---|---|---|---|---|---|---|
| 0 | FCN | 1.000000 | 1.000000 | 1.000000 | 1.000000 | 1.000000 |
| 1 | MLP | 0.813333 | 0.827160 | 0.779070 | 0.881579 | 0.901671 |
| 2 | quantile-MinCut_GCNConv | 0.900000 | 0.901961 | 0.896104 | 0.907895 | 0.967283 |
| 3 | quantile-PoC_GCNs | 0.473333 | 0.048193 | 0.285714 | 0.026316 | 0.506401 |
| 4 | quantile-TopK_GIN | 0.693333 | 0.752688 | 0.636364 | 0.921053 | 0.798898 |
| 5 | quantile-TopK_GraphConv | 0.660000 | 0.724324 | 0.614679 | 0.881579 | 0.765469 |
| 6 | ResNet | 0.993333 | 0.993464 | 0.987013 | 1.000000 | 1.000000 |
| 7 | visibility-feat | 0.806667 | 0.785185 | 0.898305 | 0.697368 | 0.904161 |
| 8 | visibility-no-feat | 0.940000 | 0.941176 | 0.935065 | 0.947368 | 0.979196 |
test_evaluator.plot_confusion_matrices_grid(plot_title="Confusion matrices for test set", cols_number_on_plot=3)
C:\Users\Ela\anaconda3\lib\site-packages\seaborn\axisgrid.py:88: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect. C:\Users\Ela\AppData\Local\Temp/ipykernel_23800/1108268709.py:132: UserWarning: Matplotlib is currently using module://matplotlib_inline.backend_inline, which is a non-GUI backend, so cannot show the figure.
test_evaluator.plot_precision_recall(width=1000, height=800)
test_evaluator.plot_roc_curves(width=1000, height=800, plot_title="ROC curve")
train_evaluator = ClassificationEvaluator(predictions=train_pred_df)
train_evaluator.calculate_classification_metrics(if_return_melted_not_pivoted=False)
| method | accuracy | f1 | precision | recall | AUC | |
|---|---|---|---|---|---|---|
| 0 | FCN | 1.00 | 1.000000 | 1.000000 | 1.000000 | 1.000000 |
| 1 | MLP | 0.92 | 0.923077 | 0.857143 | 1.000000 | 0.996795 |
| 2 | quantile-MinCut_GCNConv | 0.98 | 0.979592 | 0.960000 | 1.000000 | 1.000000 |
| 3 | quantile-PoC_GCNs | 0.52 | 0.076923 | 0.500000 | 0.041667 | 0.663462 |
| 4 | quantile-TopK_GIN | 0.70 | 0.745763 | 0.628571 | 0.916667 | 0.838141 |
| 5 | quantile-TopK_GraphConv | 0.70 | 0.745763 | 0.628571 | 0.916667 | 0.806090 |
| 6 | ResNet | 1.00 | 1.000000 | 1.000000 | 1.000000 | 1.000000 |
| 7 | visibility-feat | 0.94 | 0.938776 | 0.920000 | 0.958333 | 0.977564 |
| 8 | visibility-no-feat | 0.92 | 0.920000 | 0.884615 | 0.958333 | 0.975962 |
train_evaluator.plot_confusion_matrices_grid(plot_title="Confusion matrices for train set", cols_number_on_plot=3)
C:\Users\Ela\anaconda3\lib\site-packages\seaborn\axisgrid.py:88: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect. C:\Users\Ela\AppData\Local\Temp/ipykernel_23800/1108268709.py:132: UserWarning: Matplotlib is currently using module://matplotlib_inline.backend_inline, which is a non-GUI backend, so cannot show the figure.
train_evaluator.plot_precision_recall(width=1000, height=800)
train_evaluator.plot_roc_curves(width=1000, height=800, plot_title="ROC curve")